# -*- coding: utf-8 -*-
import codecs

def clean_sx(line):
    sx = u'ςχ'
    if sx in line:
        sx_idx = line.index(sx)+1
        dash_idx = line.index('-')
        s1=line[:sx_idx]+line[dash_idx:]
        s2=line[sx_idx:]
        return[s1,s2]
    return None

def clean_nx(line):
    sx = u'νχ'
    if sx in line:
        sx_idx = line.index(sx)+1
        dash_idx = line.index('-')
        s1=line[:sx_idx]+line[dash_idx:]
        s2=line[sx_idx:]
        return[s1,s2]
    return None

def clean_nax(line):
    nax = u'ναχ'
    if nax in line and line.startswith(nax):
        sx_idx = line.index(nax)+1
        dash_idx = line.index('-')
        s1=line[:sx_idx]+line[dash_idx:]
        s2=line[sx_idx:]
        return[s1,s2]
    return None

def clean_enax(line):
    nax = u'εναχ'
    stenax = u'στεναχ'
    if nax in line and stenax not in line:
        sx_idx = line.index(nax)+1
        dash_idx = line.index('-')
        s1=line[:sx_idx]+line[dash_idx:]
        s2=line[sx_idx:]
        return[s1,s2]
    return None

def clean_enax2(line):
    nax = u'έναχ'
    stenax = u'στέναχ'
    if nax in line and stenax not in line:
        sx_idx = line.index(nax)+1
        dash_idx = line.index('-')
        s1=line[:sx_idx]+line[dash_idx:]
        s2=line[sx_idx:]
        return[s1,s2]
    return None

def clean_words(filename):
    funcs = (clean_sx,clean_nx,clean_nax,clean_enax,clean_enax2)
    file = codecs.open(filename, mode = 'r', encoding='utf-8')
    file_write = codecs.open('clean_'+filename, mode = 'wb', encoding='utf-8')
    for line in file:
        write = True
        for func in funcs:
            l = func(line)
            if l:
                file_write.write(l[0])
                file_write.write(l[1])  
                write=False         
        if write:
            file_write.write(line)
    file_write.flush()
    file_write.close()
    
from utils import util
import re


def remove_bad_words(filename,bad_words):
    file = codecs.open(filename, mode = 'r', encoding='utf-8')
    bad_file = codecs.open(bad_words, mode = 'r', encoding='utf-8')
    file_write = codecs.open('clean_'+filename, mode = 'wb', encoding='utf-8')
    bad = []
    for word in bad_file:
        bad.append(word)
        
    pattern = re.compile(r'(\w+) - (\w+) - (\d+) - ', re.UNICODE)
    
    for line in file:
        m = pattern.match(line)
        if m == None: continue
        grkl_word = m.group(1)
        if not grkl_word+'\r\n' in bad:
            file_write.write(line)
        else:
            print 'Removing:',line.encode('utf-8')
    file_write.flush()
    file_write.close()
        
    


def test_clean():
    s1 = u"συγκεκριμένεςχώρες-63-(2.98671197619e-006)"
    s2 = u"συζύγους-63-(2.98671197619e-006)"
    assert clean_sx(s1)==[u'συγκεκριμένες-63-(2.98671197619e-006)',u'χώρες-63-(2.98671197619e-006)']
    assert clean_sx(s2)==None  
    print s1.encode('utf-8'),clean_sx(s1)[0].encode('utf-8'),clean_sx(s1)[1].encode('utf-8')
    print s2.encode('utf-8'),clean_sx(s2)
    
if __name__ == '__main__':
#    clean_words('clean_greekwords2.txt')
    remove_bad_words('word_list.txt','bad_words.txt')